HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1import type { Metadata } from 'next';2import Link from 'next/link';3import { AdminTitle, JsonPre, KindChip, Mono, Notice, StatusChip } from '@/components/admin/ui';4import { SectionNav } from '@/components/layout/terminal';5import { EntityBadge, TierBadge } from '@/components/ui/badges';6import { DataTable, EmptyRow, Td, Th } from '@/components/ui/data-table';7import { Note } from '@/components/ui/section';8import { Unavailable } from '@/components/ui/unavailable';9import { adminApi, load, requireAdmin } from '@/lib/admin/admin-api';10import type { ExtractionPayload, ExtractionSpan } from '@/lib/admin/types';11import { cn } from '@/lib/cn';12import { fmtBytes, fmtDateTime, fmtInt, fmtValue, num } from '@/lib/format';13import { routes } from '@/lib/site';1415export const metadata: Metadata = { title: 'Extraction debugger', robots: { index: false, follow: false } };16export const dynamic = 'force-dynamic';1718const STAGES = [19 { id: 'raw', label: 'Raw' },20 { id: 'normalized', label: 'Normalized' },21 { id: 'deterministic', label: 'Deterministic' },22 { id: 'llm', label: 'LLM' },23 { id: 'candidates', label: 'Candidates' },24 { id: 'claims', label: 'Claims' },25 { id: 'relations', label: 'Relations' },26 { id: 'reconciliation', label: 'Reconciliation' },27 { id: 'events', label: 'Events' },28];2930/** Text excerpt with the located claim values highlighted (offsets are into the cleaned text). */31function HighlightedText({ text, spans }: { text: string; spans: ExtractionSpan[] }) {32 const found = spans.filter((s) => s.found && typeof s.offset === 'number' && s.offset >= 0 && s.offset < text.length && s.match).sort((a, b) => (a.offset ?? 0) - (b.offset ?? 0));33 const parts: React.ReactNode[] = [];34 let cursor = 0;35 for (const s of found) {36 const start = s.offset as number;37 const len = (s.match as string).length;38 if (start < cursor) continue;39 parts.push(text.slice(cursor, start));40 parts.push(41 <mark key={s.claim_id} id={`span-${s.claim_id}`} className="rounded-[2px] bg-accent-soft px-0.5 text-ink outline outline-1 outline-accent/40" title={`${s.property} = ${fmtValue(s.value, s.property)}`}>42 {text.slice(start, start + len)}43 </mark>,44 );45 cursor = start + len;46 }47 parts.push(text.slice(cursor));48 return <>{parts}</>;49}5051function Stage({ id, title, count, children, lede }: { id: string; title: string; count?: number | null; children: React.ReactNode; lede?: string }) {52 return (53 <section id={id} className="scroll-mt-24 border-t border-rule py-5">54 <h2 className="text-base font-semibold tracking-tight">55 {title} {count !== undefined && <span className="tnum text-sm font-normal text-ink-3">{fmtInt(count)}</span>}56 </h2>57 {lede && <p className="mt-0.5 text-xs text-ink-3">{lede}</p>}58 <div className="mt-3">{children}</div>59 </section>60 );61}6263function ClaimsTable({ claims, spans, caption }: { claims: ExtractionPayload['claims']; spans: ExtractionSpan[]; caption: string }) {64 const spanById = new Map(spans.map((s) => [s.claim_id, s]));65 return (66 <DataTable compact scroll caption={caption}>67 <thead>68 <tr>69 <Th>Entity</Th>70 <Th>Property</Th>71 <Th>Value</Th>72 <Th>Raw</Th>73 <Th>Status</Th>74 <Th>Conf.</Th>75 <Th>Tier</Th>76 <Th>Located in text</Th>77 <Th>Claim</Th>78 </tr>79 </thead>80 <tbody>81 {claims.length === 0 && <EmptyRow cols={9}>No claims at this stage.</EmptyRow>}82 {claims.map((c) => {83 const s = spanById.get(c.id);84 return (85 <tr key={c.id}>86 <Td primary>{c.entity_slug ? <Link href={routes.entity({ entity_type: 'model', slug: c.entity_slug })} className="text-ink hover:text-accent">{c.entity_slug}</Link> : <Mono>{c.entity_id ?? '—'}</Mono>}</Td>87 <Td>88 <Mono>{c.property}</Mono>89 </Td>90 <Td className="tnum text-xs">{fmtValue(c.value, c.property)}{c.unit ? <span className="text-ink-3"> {c.unit}</span> : null}</Td>91 <Td className="text-xs text-ink-3">{c.value_raw === null || c.value_raw === undefined ? '—' : String(c.value_raw)}</Td>92 <Td>93 <StatusChip value={c.status} />94 </Td>95 <Td className="text-xs text-ink-2">{c.confidence}</Td>96 <Td>97 <TierBadge tier={num(c.tier)} />98 </Td>99 <Td className="text-xs">100 {s?.found ? (101 <a href={`#span-${c.id}`} className="text-positive hover:underline">102 found @ {s.offset}103 </a>104 ) : s ? (105 <span className="text-warning" title={s.tried?.length ? `tried: ${s.tried.join(', ')}` : undefined}>106 not found107 </span>108 ) : (109 <span className="text-ink-3">—</span>110 )}111 </Td>112 <Td>113 <Link href={routes.claim(c.id)} className="mono text-[11px] text-accent hover:underline">114 {c.id}115 </Link>116 </Td>117 </tr>118 );119 })}120 </tbody>121 </DataTable>122 );123}124125export default async function ExtractionPage({ params, searchParams }: { params: Promise<{ snapshot: string }>; searchParams: Promise<Record<string, string | undefined>> }) {126 await requireAdmin();127 const { snapshot } = await params;128 const sp = await searchParams;129 const res = await load(adminApi.extraction(snapshot, 20000));130 if (!res.ok) {131 return (132 <>133 <AdminTitle title="Extraction debugger" />134 <Unavailable what="Extraction" reason={res.error} />135 </>136 );137 }138 const x = res.data;139 const det = x.claims.filter((c) => c.extractor !== 'llm');140 const llm = x.claims.filter((c) => c.extractor === 'llm');141 const text = x.text ?? '';142 return (143 <>144 <AdminTitle title="Extraction debugger" count={<Mono>{x.id}</Mono>} lede="The pipeline for one snapshot: raw fetch → cleaned text → deterministic claims → LLM claims → entity candidates → claims written → relations → reconciliation with the previous snapshot → events. Value locations are a best-effort search in the text; not-found is reported, never inferred.">145 <Link href={`/admin/snapshots/${encodeURIComponent(x.id)}`} className="text-xs text-ink-3 hover:text-ink">146 Snapshot record →147 </Link>148 {x.document_id && (149 <Link href={`/admin/documents/${encodeURIComponent(x.document_id)}`} className="text-xs text-ink-3 hover:text-ink">150 Document →151 </Link>152 )}153 </AdminTitle>154 <Notice notice={sp.notice} level={sp.level} />155 <SectionNav items={STAGES} className="mb-2" />156157 <Stage id="raw" title="RAW" lede="What was fetched (paths are never exposed).">158 <dl className="kv [&>div]:py-1 text-sm">159 <div>160 <dt>URL</dt>161 <dd>162 <a href={x.final_url ?? x.url} target="_blank" rel="noopener noreferrer" className="link break-all text-xs">163 {x.final_url ?? x.url}164 </a>165 </dd>166 </div>167 <div>168 <dt>Observed</dt>169 <dd className="tnum">{fmtDateTime(x.observed_at)}</dd>170 </div>171 <div>172 <dt>HTTP · type · size</dt>173 <dd className="tnum">174 {fmtValue(x.http_status)} · {x.content_type ?? '—'} · {fmtBytes(x.byte_size)}175 </dd>176 </div>177 <div>178 <dt>Hashes</dt>179 <dd>180 <Mono>content {x.content_hash?.slice(0, 16) ?? '—'}</Mono> <Mono>text {x.text_hash?.slice(0, 16) ?? '—'}</Mono>181 </dd>182 </div>183 <div>184 <dt>Connector · parser · transport</dt>185 <dd>186 <Mono>{x.connector_name ?? '—'}</Mono> <Mono>parser v{x.parser_version ?? '—'}</Mono> <Mono>{x.transport ?? '—'}</Mono>187 </dd>188 </div>189 <div>190 <dt>Run · document</dt>191 <dd>192 <Mono>{x.run_id ?? '—'}</Mono> · <Mono>{x.document_id}</Mono> {x.doc_type && <KindChip value={x.doc_type} />}193 </dd>194 </div>195 <div>196 <dt>Entity</dt>197 <dd>198 {x.entity_slug ? (199 <span className="inline-flex items-center gap-1.5">200 <EntityBadge type="model" small />201 <Link href={routes.entity({ entity_type: 'model', slug: x.entity_slug })} className="text-ink hover:text-accent">202 {x.entity_name ?? x.entity_slug}203 </Link>204 </span>205 ) : (206 '—'207 )}208 </dd>209 </div>210 <div>211 <dt>Flags</dt>212 <dd className="text-xs">213 raw {x.has_raw ? 'archived' : '—'} · text {x.has_text ? 'yes' : '—'} · structured {x.has_structured || x.structured ? 'yes' : '—'} · changed {x.changed ? 'yes' : 'no'} · <StatusChip value={x.processing_status} />214 </dd>215 </div>216 </dl>217 {x.structured != null && (218 <details className="mt-3">219 <summary className="cursor-pointer text-xs text-ink-3 hover:text-ink">Structured data (JSON-LD, OG, embedded JSON)</summary>220 <JsonPre value={x.structured} maxHeight="20rem" />221 </details>222 )}223 </Stage>224225 <Stage id="normalized" title="NORMALIZED" count={num(x.text_chars)} lede={`Cleaned text${x.text_truncated ? ' (excerpt — truncated to 20 kB)' : ''}; ${fmtInt(x.spans_found)} of ${fmtInt(x.spans.length)} claim values located and highlighted.`}>226 {text ? (227 <pre className="scrollbar-thin max-h-[32rem] overflow-auto whitespace-pre-wrap border border-rule bg-surface p-3 text-[12px] leading-relaxed text-ink-2" data-extraction-text>228 <HighlightedText text={text} spans={x.spans} />229 </pre>230 ) : (231 <p className="text-sm text-ink-3">No cleaned text{x.text_error ? ` — ${x.text_error}` : ''}.</p>232 )}233 </Stage>234235 <Stage id="deterministic" title="DETERMINISTIC" count={det.length} lede="Rule-based extraction (tables, JSON-LD, meta, regex) — always runs first.">236 <ClaimsTable claims={det} spans={x.spans} caption="Deterministic claims" />237 </Stage>238239 <Stage id="llm" title="LLM" count={llm.length + x.llm_jobs.length} lede="LLM jobs for this snapshot and the claims they produced (one tier lower than deterministic).">240 {x.llm_jobs.length > 0 && <JsonPre value={x.llm_jobs} maxHeight="14rem" />}241 {x.llm_jobs.length === 0 && llm.length === 0 ? <p className="text-sm text-ink-3">No LLM extraction for this snapshot (deterministic only).</p> : <ClaimsTable claims={llm} spans={x.spans} caption="LLM claims" />}242 </Stage>243244 <Stage id="candidates" title="CANDIDATES" count={x.entity_candidates.length} lede="Entities the resolver matched or created for this document.">245 <DataTable compact caption="Entity candidates">246 <thead>247 <tr>248 <Th>Entity</Th>249 <Th>Type</Th>250 <Th>Identity</Th>251 <Th>Merged into</Th>252 <Th>Id</Th>253 </tr>254 </thead>255 <tbody>256 {x.entity_candidates.length === 0 && <EmptyRow cols={5}>No candidate.</EmptyRow>}257 {x.entity_candidates.map((c) => (258 <tr key={c.id}>259 <Td primary>260 <Link href={routes.entity({ entity_type: c.entity_type, slug: c.slug })} className="text-ink hover:text-accent">261 {c.canonical_name}262 </Link>263 </Td>264 <Td>265 <EntityBadge type={c.entity_type} small />266 </Td>267 <Td className="text-xs text-ink-2">{c.identity_confidence ?? '—'}</Td>268 <Td>{c.merged_into ? <Mono>{c.merged_into}</Mono> : <span className="text-ink-3">—</span>}</Td>269 <Td>270 <Mono>{c.id}</Mono>271 </Td>272 </tr>273 ))}274 </tbody>275 </DataTable>276 </Stage>277278 <Stage id="claims" title="CLAIMS" count={x.claims.length} lede="Every claim written from this snapshot, with its status after the writer's temporal rules.">279 <ClaimsTable claims={x.claims} spans={x.spans} caption="All claims" />280 {(x.results.length > 0 || x.prices.length > 0) && (281 <div className="mt-4 grid gap-4 md:grid-cols-2">282 <div>283 <p className="eyebrow mb-1">Benchmark results {fmtInt(x.results.length)}</p>284 <JsonPre value={x.results} maxHeight="14rem" />285 </div>286 <div>287 <p className="eyebrow mb-1">Prices {fmtInt(x.prices.length)}</p>288 <JsonPre value={x.prices} maxHeight="14rem" />289 </div>290 </div>291 )}292 </Stage>293294 <Stage id="relations" title="RELATIONS" count={x.relations.length}>295 {x.relations.length ? <JsonPre value={x.relations} maxHeight="18rem" /> : <p className="text-sm text-ink-3">No relation written from this snapshot.</p>}296 </Stage>297298 <Stage id="reconciliation" title="RECONCILIATION" lede="Against the previous snapshot of the same document: what changed, and how the writer treated each claim (confirm / supersede / conflict).">299 {x.previous_snapshot ? (300 <dl className="kv [&>div]:py-1 text-sm">301 <div>302 <dt>Previous snapshot</dt>303 <dd>304 <Link href={`/admin/extractions/${encodeURIComponent(x.previous_snapshot.id)}`} className="mono text-xs text-accent hover:underline">305 {x.previous_snapshot.id}306 </Link>{' '}307 <span className="tnum text-xs text-ink-3">{fmtDateTime(x.previous_snapshot.observed_at)}</span>308 </dd>309 </div>310 <div>311 <dt>Content changed</dt>312 <dd className="text-xs">{x.previous_snapshot.content_hash && x.content_hash ? (x.previous_snapshot.content_hash === x.content_hash ? 'no (same hash)' : 'yes (hash differs)') : '—'}</dd>313 </div>314 <div>315 <dt>Claim outcomes</dt>316 <dd className="text-xs">317 {['current', 'superseded', 'conflicting', 'retracted'].map((s) => (318 <span key={s} className={cn('mr-3 tnum', s === 'conflicting' && x.claims.some((c) => c.status === s) && 'text-danger')}>319 {s} {fmtInt(x.claims.filter((c) => c.status === s).length)}320 </span>321 ))}322 </dd>323 </div>324 </dl>325 ) : (326 <p className="text-sm text-ink-3">First snapshot of this document — nothing to reconcile against.</p>327 )}328 {x.diff != null && (329 <details className="mt-3">330 <summary className="cursor-pointer text-xs text-ink-3 hover:text-ink">Diff payload</summary>331 <JsonPre value={x.diff} maxHeight="18rem" />332 </details>333 )}334 </Stage>335336 <Stage id="events" title="EVENTS" count={x.events.length} lede="Change events emitted by this snapshot (material properties only).">337 {x.events.length ? <JsonPre value={x.events} maxHeight="18rem" /> : <p className="text-sm text-ink-3">No event emitted.</p>}338 </Stage>339 {x.note && <Note className="mt-4">{x.note}</Note>}340 </>341 );342}343